# -*- coding: utf-8 -*-
# @Time : 2022/8/5 下午2:53
# @Author : WangYunfei
# @FileName: demo7.py
# @Email: yunfei.wang@innvote.com
"""
7. 使用网络爬虫技术采集该站点10000条数据，且需要设计data去重算法，使用线程池采集 ,字段不限，mongo存储 ,
采集地址：www.oklink.com/zh-cn/btc/tx-list 。交付：提供数据和代码截图即可。说明：禁止使用自动化等技术
"""
import asyncio
import json
import time

import aiohttp
import execjs
import requests
from aiohttp import ClientTimeout
from gevent.threadpool import ThreadPoolExecutor

from base import logger
from example.kaoshi.demo6 import ZBMongoDB


class BTCMongoDB(ZBMongoDB):
    """ BTC """


class BTC(object):

    def __init__(self, total=10000):
        self.total = total
        self.url = "https://www.oklink.com/api/explorer/v1/btc/transactionsNoRestrict"
        self.mongo = BTCMongoDB("btc", db="kaoshi")

    @staticmethod
    def headers(t):
        return {
            "x-apiKey": execjs.compile(open("./demo7.js").read()).call("getApiKey", t),
        }

    async def _get_data(self, params):
        # res = requests.get(url=self.url, params=params, headers=self.headers(params["t"]))
        # print(res.text)
        timeout = ClientTimeout(total=30)
        async with aiohttp.ClientSession(timeout=timeout) as session:
            try:
                async with session.get(self.url, params=params, headers=self.headers(params["t"])) as res:
                    if res.status != 200:
                        logger.warning(f'请求错误{res.status}--')
                    else:
                        body = await res.read()
                        body = body.decode()  # 对读回来的原始字节解码
                        _btc_data = json.loads(body)["data"]["hits"]
                        for item in _btc_data:
                            _btc = {
                                "_id": self.mongo.md5_str(item["hash"]),
                                "hash": item["hash"],
                                "所在区块": item["blockHeight"],
                                "时间": item["blocktime"],
                                "数量(BTC)": item["inputsValue"]
                            }
                            self.mongo.insert_one(_btc)
                        res.close()
            except asyncio.TimeoutError as e:
                logger.warning(f'请求超时错误')

    def get_data(self, params):
        asyncio.run(self._get_data(params))
        # loop = asyncio.get_event_loop()
        # loop.run_until_complete(self._get_data(params))

    def run(self):
        page_size = 100
        with ThreadPoolExecutor(10) as f:
            for i in range(0, self.total, page_size):
                params = {
                    "t": int(time.time() * 1000),
                    "limit": page_size,
                    "offset": i
                }
                f.submit(self.get_data, params=params)


if __name__ == '__main__':
    BTC().run()




